# 爬取同花顺概念信息_ 头部已分类信息
import os
import zipfile

import requests
import pymysql
import traceback
import undetected_chromedriver as uc
from lxml import etree
from selenium.common import NoSuchElementException
from selenium.webdriver.common.by import By
import time
from dbse.mysql.db_mysql_insert import addStockThsSelException
from dbse.mysql.db_entity_stock_ths_exception import StockThsSelException
from dbse.mysql.db_mysql_insert import addThsGNStocks
from dbse.mysql.db_mysql_insert import addStockKpi
from dbse.mysql.db_mysql_insert import updateGnCompleted
from dbse.mysql.db_mysql_insert import deleteGnStocks
from dbse.mysql.db_entity_stock_ths_gn_stocks import StockThsGNStocks
from dbse.mysql.db_entity_stock_kpi_day import StockKpiDay
from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from sel.proxy.selenium_proxy_auth_plugin import create_proxy_auth_extension as cpae
from utils.StringUtil import simpleReplace
from utils.DateUtil import getTodayStr
from selenium.webdriver.support import expected_conditions as EC
from file.readFile import read
import datetime
import json


def addException(db,stockCode):
    sql = 'INSERT INTO stock_ths_sel_exception (biz_code,sel_type,sta_date) VALUES (%s,%s,%s)'
    entity = StockThsSelException(stockCode,'1',getTodayStr())
    addStockThsSelException(db,sql,entity)

# 处理一页数据
def sel_gn_stocks(browser,db,add_sql,gn):
    gnCode = gn[0]                                              # 概念代码
    today = datetime.date.today().strftime("%Y-%m-%d")          # 时间
    content = browser.page_source
    # print(content)
    html = etree.fromstring(content,etree.HTMLParser())
    # result = html.xpath('//tbody//tr')                          # 股票数据
    result = html.xpath('//div[@id="maincont"]//tbody//tr')       # 股票数据 注意: 网页有多个版本 有的网页有两个tbody 所有要再通过div id=maincont 过滤一层
    for index, tr in enumerate(result):
        tds = tr.getchildren()
        # splitUrlItems = tds[1].find('a').get('href').split("/")
        # gnCode = splitUrlItems[len(splitUrlItems)-2]
        # print(f"====index:{index}")
        # print(tds[1].find('a').text)                # 股票代码
        # print(tds[2].find('a').text)                # 股票名称
        entity = StockThsGNStocks(tds[1].find('a').text,tds[2].find('a').text,today,gnCode)
        addThsGNStocks(db,add_sql,entity)

# 获取代理server 地址 有效期1-5分钟（私密代理）
def get_proxy():
    url = "https://dps.kdlapi.com/api/getdps/?secret_id=ondbxg8msovv7ury9qbe&signature=zft63syjut8u3og5m9j5wung50mwah5f&num=1&format=text&sep=1"
    proxy_address = requests.get(url)
    return proxy_address.text

def getPage(browser,stockCode,jys):
    url = f"https://quote.eastmoney.com/{jys}{stockCode}.html"
    browser.get(url)

def initChrome():
    ### 初始化chrome浏览器 和 cookie
    driver_path = r'C:\Program Files\Google\Chrome\Application\chromedriver.exe'
    useProxy = False
    if useProxy:
        option = ChromeOptions()
        proxy_address = get_proxy()
        print(f"===============使用代理:{proxy_address}")
        proxy_host = proxy_address.split(":")[0]            # proxy ip
        proxy_port = proxy_address.split(":")[1]            # proxy port
        proxy_username = "d2231274096"                      # 代理账号
        proxy_password = "ela9v3e9"                         # 代理密码
        extension_path = cpae(proxy_host, proxy_port, proxy_username, proxy_password)   # 配置代理插件

        option.add_extension(extension_path)
        option.add_argument('--ignore-certificate-errors')
        option.add_argument('--disable-web-security')
        option.add_argument('--allow-running-insecure-content')
        option.add_argument('--no-sandbox')
        option.add_argument('--disable-dev-shm-usage')
        browser = uc.Chrome(driver_executable_path=driver_path,options=option)        # 生产用: 走代理 会消耗 ip
    else:
        option = ChromeOptions()
        option.add_argument('--headless')
        browser = uc.Chrome(driver_executable_path=driver_path,options=option)                         # 测试用: 不使用代理 适合跑单个概念

    browser.set_window_size(1366,768)
    # # 定义cookie 需要先通过get_cookie.py 生成cookie.txt
    # cookies = []
    # with open('./cookie/cookie.txt', 'r') as f:
    #     cookies = json.load(f)
    # # 先访问一下主页 让browser先识别域名
    # url = f"https://quote.eastmoney.com/{jys}{stockCode}.html"
    # browser.get(url)
    browser.get("https://quote.eastmoney.com")
    # time.sleep(2)       # 让页面加载一下
    # # 加载cookie
    # for cookie in cookies:
    #     # 调整域名为当前域名 (如果域名不匹配会报错)
    #     if '.10jqka.com.cn' == cookie['domain']:
    #         # 如果cookie的域名和当前browser不匹配会报错
    #         browser.add_cookie(cookie)
    #         continue
    # # 刷新使Cookie生效
    # browser.refresh()
    return browser

def startWork():
    db = pymysql.connect(host='39.98.82.109',user='root',password='landleaf@123.com',port=3306,db='stock')

    ### 查询所有概念  先判断是否还有未完成任务
    # query_sql = 'select * from stock_all t1 where t1.stock_code in ("688075")'
    query_sql = 'select * from stock_all t1 where t1.jys ="SZ" '
    add_sql = 'INSERT INTO stock_kpi_day (stock_code, stock_name, price_begin, price_end, price_max, price_min, price_increase, price_increase_rate, per, pbr,  sta_date, turnover_ratio, amount, total_capital, trunover, price_yesterday, vol_ratio) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
    cursor = db.cursor()
    cursor.execute(query_sql)
    stockAlls = cursor.fetchall()
    if len(stockAlls) == 0:
        print("=====================没有跟多需要处理的任务了=====================")
        return

    # 初始化
    browser = initChrome()

    for stock in stockAlls:
        print(f"====处理股票{stock[1]}")
        stockCode = stock[1]
        stockName = stock[3]
        jys = stock[4].lower()

        try:
            #抓取数据
            getPage(browser,stockCode,jys)
            content = browser.page_source
            html = etree.fromstring(content,etree.HTMLParser())
            # 判断是否已退市
            error_state =  html.xpath('//div[@class="errorstate"]')
            if error_state:
                if error_state[0].text == '已退市':
                    print(f"==========股票{stockCode}:已退市")
                    continue
            # 判断是否科创板
            if stockCode.startswith("688"):
                # print(f"==========股票{stockCode}:是科创板")
                shoupan_result = html.xpath('//div[@class="zxj"]//span')
                zhangdie_result = html.xpath('//div[@class="zd"]/span')       # 两层span 嵌套 <span><span>xxx</span></span> 所以使用单 / 直接子元素
                # print(shoupan_result[0].find('span').text)                    #收盘
                # print(zhangdie_result[0].find('span').text)                   #涨跌
                # print(zhangdie_result[1].find('span').text)                   #涨跌百分比
                result = html.xpath('//div[@class="brief_info_c"]//tbody//tr')          #
                row1_tds = result[0].getchildren()
                row2_tds = result[1].getchildren()
                row3_tds = result[2].getchildren()
                # print(row1_tds[0].text)
                # print(row1_tds[5].find('span').find('span').text)           #今开
                # print(row1_tds[3].find('span').find('span').text)           #最高
                # print(row1_tds[7].find('span').find('span').text)           #换手
                # print(row1_tds[9].find('span').find('span').text)           #总市值
                # print(row2_tds[1].find('span').find('span').text)           #成交量
                # print(row2_tds[3].find('span').find('span').text)           #最低
                # print(row2_tds[5].find('span').find('span').text)           #昨收
                # print(row2_tds[7].find('span').find('span').text)           #量比
                # print(row2_tds[9].find('span').find('span').text)           #流通市值
                # print(row3_tds[1].find('span').find('span').text)           #成交额
                # print(row3_tds[3].find('span').find('span').text)           #震幅 （暂时不用）
                # print(row3_tds[7].find('span').find('span').text)           #市盈率
                # print(row3_tds[9].find('span').find('span').text)           #市净率
                stockKpiDay = StockKpiDay(stockCode,stockName,
                                          row1_tds[5].find('span').find('span').text,
                                          shoupan_result[0].find('span').text,
                                          row1_tds[3].find('span').find('span').text,
                                          row2_tds[3].find('span').find('span').text,
                                          zhangdie_result[0].find('span').text,
                                          simpleReplace(zhangdie_result[1].find('span').text),
                                          row3_tds[7].find('span').find('span').text,
                                          row3_tds[9].find('span').find('span').text,
                                          getTodayStr(),
                                          simpleReplace(row1_tds[7].find('span').find('span').text),
                                          simpleReplace(row3_tds[1].find('span').find('span').text),
                                          simpleReplace(row1_tds[9].find('span').find('span').text),
                                          simpleReplace(row2_tds[1].find('span').find('span').text),
                                          row2_tds[5].find('span').find('span').text,
                                          row2_tds[7].find('span').find('span').text
                                          )
                addStockKpi(db,add_sql,stockKpiDay)
            else:
                shoupan_result = html.xpath('//div[@class="zxj"]//span')
                zhangdie_result = html.xpath('//div[@class="zd"]/span')       # 两层span 嵌套 <span><span>xxx</span></span> 所以使用单 / 直接子元素
                # print(shoupan_result[0].find('span').text)                    #收盘
                # print(zhangdie_result[0].find('span').text)                   #涨跌
                # print(zhangdie_result[1].find('span').text)                   #涨跌百分比

                result = html.xpath('//div[@class="brief_info_c"]//tbody//tr')          #
                row1_tds = result[0].getchildren()
                row2_tds = result[1].getchildren()
                # print(row1_tds[0].text)
                # print(row1_tds[1].find('span').find('span').text)           #今开
                # print(row1_tds[3].find('span').find('span').text)           #最高
                # print(row1_tds[5].find('span').find('span').text)           #涨停
                # print(row1_tds[7].find('span').find('span').text)           #换手
                # print(row1_tds[9].find('span').find('span').text)           #成交量
                # print(row1_tds[11].find('span').find('span').text)           #市盈(动)
                # print(row1_tds[13].find('span').find('span').text)           #总市值
                #
                # print(row2_tds[1].find('span').find('span').text)           #昨收
                # print(row2_tds[3].find('span').find('span').text)           #最低
                # print(row2_tds[5].find('span').find('span').text)           #跌停
                # print(row2_tds[7].find('span').find('span').text)           #量比
                # print(row2_tds[9].find('span').find('span').text)           #成交额
                # print(row2_tds[11].find('span').find('span').text)           #市净
                # print(row2_tds[13].find('span').find('span').text)           #流通市值

                stockKpiDay = StockKpiDay(stockCode,stockName,
                                          row1_tds[1].find('span').find('span').text,
                                          shoupan_result[0].find('span').text,
                                          row1_tds[3].find('span').find('span').text,
                                          row2_tds[3].find('span').find('span').text,
                                          zhangdie_result[0].find('span').text,
                                          simpleReplace(zhangdie_result[1].find('span').text),
                                          row1_tds[11].find('span').find('span').text,
                                          row2_tds[11].find('span').find('span').text,
                                          getTodayStr(),
                                          simpleReplace(row1_tds[7].find('span').find('span').text),
                                          simpleReplace(row2_tds[9].find('span').find('span').text),
                                          simpleReplace(row1_tds[13].find('span').find('span').text),
                                          simpleReplace(row1_tds[9].find('span').find('span').text),
                                          row2_tds[1].find('span').find('span').text,
                                          row2_tds[7].find('span').find('span').text
                                          )
                addStockKpi(db,add_sql,stockKpiDay)
        except Exception as e:
            # print(f"异常信息: {e}")
            # print("详细堆栈信息:")
            # traceback.print_exc()  # 打印完整的堆栈跟踪
            print(f"======================{stockCode}发生异常======================")
            addException(db,stockCode)
            time.sleep(1)

for index in range(1):
    print(f"=================批次{index}完成，等待下一个批次===================")
    startWork()






